package com.example.webmagic;

import org.apache.commons.io.FileUtils;
import sun.security.pkcs11.wrapper.Constants;
import us.codecraft.webmagic.*;
import us.codecraft.webmagic.pipeline.ConsolePipeline;
import us.codecraft.webmagic.pipeline.Pipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.QueueScheduler;
import us.codecraft.webmagic.scheduler.component.HashSetDuplicateRemover;

import java.io.*;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class Crawl_Video implements PageProcessor {
    private Site site = Site.me().setRetryTimes(3).setSleepTime(1000).setTimeOut(10000);

    @Override
    public void process(Page page) {
        List<String> list = new ArrayList<>();
        List<String> all = page.getHtml().xpath("//video[@src]/@src").all();
        List<String> all1 = page.getHtml().xpath("//video//[@src]/@src").all();
        list.addAll(all);
        list.addAll(all1);

        page.putField("urlString",list);

        page.putField("urltitle",page.getUrl());
        System.out.println(list);

    }

    @Override
    public Site getSite() {
        return site;
    }


    public static void main(String[] args) {


        Spider.create(new Crawl_Video()).addUrl("http://www.djjhj.com/")
                .addPipeline(new ConsolePipeline())//打印至控制台
                .addPipeline(new Video_Pipeline())
                .thread(50).run();
    }


    public static class Video_Pipeline implements Pipeline {

        @Override
        public void process(ResultItems resultItems, Task task) {
            long l = System.currentTimeMillis();
            List<String> urlString = resultItems.get("urlString");
            String urltitle = resultItems.get("urltitle").toString();


            // 构造URL
            try {

                for (int i = 0; i < urlString.size(); i++) {
                    String urlStr= "";
                    if (!urlString.get(i).startsWith("http")){
                        urlStr=urltitle+urlString.get(i);
                    }else{
                        urlStr=urlString.get(i);
                    }

                    URL url = new URL(urlStr);
                    String host = url.getHost();


                    String type = urlString.get(i).substring(urlString.get(i).lastIndexOf("."));
                    // 打开连接
                    URLConnection con = url.openConnection();
                    //设置请求超时为20s
                    con.setConnectTimeout(5 * 1000);
                    //文件路径不存在 则创建
                    /* File file = new File("D:\\web\\视频\\"+host+type);*/
                    File file = new File("D:\\web\\视频\\");
                   /* if (!file.exists()) {
                        file.mkdir();
                    }*/

                    /*FileUtils.copyURLToFile(url, file);*/


                    //输入流

                    InputStream in = con.getInputStream();
                    //输出流
                    OutputStream out = new FileOutputStream(file.getPath() + "\\" + host + type);

                    //创建缓冲区
                    byte[] buff = new byte[1024 * 100];
                    int n;
                    // 开始读取
                    while ((n = in.read(buff)) >= 0) {
                        out.write(buff, 0, n);
                    }
                    in.close();
                    out.close();
                    System.out.println(System.currentTimeMillis() - l);
                }
            } catch (Exception e) {
                /* e.printStackTrace();*/
            }
        }
    }
}
